package org.study.lucene.api.analyzer;

import com.google.common.collect.Sets;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.study.lucene.api.constant.PathConstants;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * 类的描述
 *
 * @author Administrator
 * @date 2021-12-30
 */
public class AnalyzerTest {
    public static void main(String[] args) {
        // org.apache.lucene.analysis.StopAnalyzer 类的常量
        String[] ENGLISH_STOP_WORDS = new String[]{"a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"};
        String[] CHINESE_STOP_WORDS = {"着", "的", "之", "式"};

        CharArraySet charArraySet = new CharArraySet(Sets.newHashSet(CHINESE_STOP_WORDS), true);
        File file = new File(PathConstants.DATA_PATH + "\\test.txt");

        try (Reader reader = new FileReader(file);
             FileReader stopWords = new FileReader(PathConstants.DATA_PATH + "\\stopWords.txt");
             //Analyzer analyzer = new StandardAnalyzer();
             //Analyzer analyzer = new StandardAnalyzer(charArraySet);
             Analyzer analyzer = new StandardAnalyzer(stopWords);
             //Analyzer analyzer = new CJKAnalyzer();
             //Analyzer analyzer = new WhitespaceAnalyzer();
             TokenStream tokenStream = analyzer.tokenStream("", reader)
        ) {
            tokenStream.reset();
            CharTermAttribute teAttribute = tokenStream.getAttribute(CharTermAttribute.class);
            List<String> tokens = new ArrayList<>();
            while (tokenStream.incrementToken()) {
                tokens.add(teAttribute.toString());
            }
            System.out.println("分词结果：" + tokens);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
